# %%
import datetime as dt
import numpy as np
import pandas as pd

# %%
from zipfile import ZipFile
# zf = ZipFile('~/coin/*.zip')
import os

file_list = []
path = 'data.usdc'
for  root, dirs, files in os.walk(path):
    for file_path in files:
        # print(file_path)
        # 获取文件的完整路径
        if  file_path.endswith('.zip'):
            file = os.path.join(root, file_path)
            print(f"文件路径: {file}")
            file_list.append(file)


cols = ['open_time', 'open', 'high', 'low', 'close', 'volume','close_time', 'quote_asset_volume','trades_num','tbbv','tbqv','ignore']
# files
# %%
# to year-month
years=["2017","2018","2019", "2020", "2021", "2022", "2023", "2024", "2025"]
years=["2020", "2021", "2022", "2023", "2024", "2025"]
months=["01","02","03","04","05","06","07","08","09","10","11","12"]
year_month = [ year+'-'+month for year in years for month in months]
import zipfile as zf
dfs = pd.DataFrame()
# results = []
temp = pd.DataFrame()
for year in year_month:
    dfs = pd.DataFrame()
    # one = pd.DataFrame()
    for text_file in file_list:
        if text_file.find(year)>0 and text_file.find('USDC')>0:
            zf = ZipFile(text_file)
            with zf.open(os.path.split(text_file)[-1].split('.')[0]+'.csv') as csv:
                # results.append(pd.read_csv(csv))
                temp = pd.read_csv(csv,header=None, names=cols)
                temp = temp[['open_time', 'open', 'high', 'low', 'close', 'volume','close_time', 'quote_asset_volume','trades_num']]
                temp['pair'] = text_file.split('-')[0].split('/')[-1].replace('USDC','/USDC')
                if dfs.empty:
                    dfs = temp
                else:
                    dfs = pd.concat([dfs,temp])

    # dfs.to_parquet(f'coin.{year}.parquet')
    dfs.to_csv(f'coin.{year}.csv')
# %%

# import pandas as pd
# import os

# # 定义包含 Parquet 文件的文件夹路径
# root = 'data.usdc/'

# q_map = {
#     '01':'1','02':'1','03':'1',
#     '04':'2','05':'2','06':'2',
#     '07':'3','08':'3','09':'3',
#     '10':'4','03':'4','04':'4'
# }
# for year in years:
#     folder_path = root + str(year)

#     for f in os.listdir(folder_path):
#         for key,value  in q_map:
#             if f.endswith('.parquet') and f.find(key)>0:
#                 # 获取文件夹中所有 Parquet 文件
#                 parquet_files = os.path.join(folder_path, f) #[os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.parquet')]

#     # 读取每个 Parquet 文件并合并到一个 DataFrame 中
#     combined_df = pd.concat([pd.read_parquet(file) for file in parquet_files], ignore_index=True)

#     # 将合并后的 DataFrame 写入一个新的 Parquet 文件
#     combined_df.to_parquet(f'{year}.parquet')
# %%



import pandas as pd
import os

# 定义包含 Parquet 文件的文件夹路径
root = 'data.usdc/'
root = '/kaggle/input/2020-2025-main-usdc/'
cols = ['time', 'open', 'high', 'low', 'close', 'volume']
# files
# %%
# to coin.year-q
years=["2017","2018","2019", "2020", "2021", "2022", "2023", "2024", "2025"]
years=["2020", "2021", "2022", "2023", "2024"]#, "2025"]
months=["01","02","03","04","05","06","07","08","09","10","11","12"]
year_month = [ year+'-'+month for year in years for month in months]
# q_map = {
#     '01':'1','02':'1','03':'1',
#     '04':'2','05':'2','06':'2',
#     '07':'3','08':'3','09':'3',
#     '10':'4','03':'4','04':'4'
# }
# for year in years:
#     folder_path = root + str(year)

#     parquet_files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.parquet')]

#     # 读取每个 Parquet 文件并合并到一个 DataFrame 中
#     combined_df = pd.concat([pd.read_parquet(file) for file in parquet_files], ignore_index=True)

#     # 将合并后的 DataFrame 写入一个新的 Parquet 文件
#     combined_df.to_parquet(f'{year}.parquet')




q_map = {
    '01':'1','02':'1','03':'1',
    '04':'2','05':'2','06':'2',
    '07':'3','08':'3','09':'3',
    '10':'4','11':'4','12':'4'
}
root = './'
for year in years:
    folder_path = root + 'csv/'  #str(year)

    print('year',year)
    for i in range(1,5):
        print('start',i)
        parquet_files = []
        for key  in q_map:
            value = q_map[key]
            if value == str(i):
                
                file_key = f'{year}-{key}'
                print('find',key,value)
                files = os.listdir(folder_path)
                files.sort()
                for f in files:
                    # if f.endswith('.parquet') and f.find(file_key)>0:
                    if  f.find(file_key)>0:
                        # 获取文件夹中所有 Parquet 文件
                        print('find file',f)
                        parquet_files.append(os.path.join(folder_path, f)) #[os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith('.parquet')]
                        break

        # 读取每个 Parquet 文件并合并到一个 DataFrame 中
        # combined_df = pd.concat([pd.read_parquet(file) for file in parquet_files], ignore_index=True)
        combined_df = pd.concat([pd.read_csv(file) for file in parquet_files], ignore_index=True)
        print('load end')
        # 将合并后的 DataFrame 写入一个新的 Parquet 文件
        # combined_df.to_parquet(f'{year}-{i}.parquet')
        combined_df.to_csv(f'{year}-{i}.csv')
        print('save end',i)
# %%
